import pymysql
from lxml import etree
import requests
import json
import re
import time
from walmart.字段 import words
import random
import parsel
from walmart.selenium_walmart import yidong

conn = pymysql.connect(host='gz-cdb-ezvn7csl.sql.tencentcdb.com', port=58892, user='root', password='shengye2021',
                       database='bailun_bltpro', charset='utf8')
cursor = conn.cursor()


def maga():
    headers = {
        'Host': 'www.walmart.com',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
        'sec-ch-ua-mobile': '?0',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Service-Worker-Navigation-Preload': 'true',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Dest': 'document',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
        'Cookie': 'AID=wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1637050244515; ACID=e57a8b4c-df53-4682-85a9-a0f6cf417ed1; hasACID=true; locGuestData=eyJpbnRlbnQiOiJTSElQUElORyIsInN0b3JlSW50ZW50IjoiUElDS1VQIiwibWVyZ2VGbGFnIjpmYWxzZSwicGlja3VwIjp7Im5vZGVJZCI6IjMwODEiLCJ0aW1lc3RhbXAiOjE2MzcwNTAyNDQ2MzJ9LCJwb3N0YWxDb2RlIjp7InRpbWVzdGFtcCI6MTYzNzA1MDI0NDYzMiwiYmFzZSI6Ijk1ODI5In0sInZhbGlkYXRlS2V5IjoicHJvZDp2MjplNTdhOGI0Yy1kZjUzLTQ2ODItODVhOS1hMGY2Y2Y0MTdlZDEifQ==; vtc=YeDTYcBVVJrmBJn6nfVFmc; TBV=7; _pxvid=b2839e3b-46b4-11ec-a6e3-585a4b527450; assortmentStoreId=3081; pxcts=3ddb24a0-4e68-11ec-ba30-89fbf66c1aed; hasLocData=1; tb_sw_supported=true; adblocked=false; dimensionData=969; TB_Latency_Tracker_100=1; TB_Navigation_Preload_01=1; TB_SFOU-100=1; locDataV3=eyJpbnRlbnQiOiJTSElQUElORyIsInBpY2t1cCI6W3siYnVJZCI6IjAiLCJub2RlSWQiOiIzMDgxIiwiZGlzcGxheU5hbWUiOiJTYWNyYW1lbnRvIFN1cGVyY2VudGVyIiwibm9kZVR5cGUiOiJTVE9SRSIsImFkZHJlc3MiOnsicG9zdGFsQ29kZSI6Ijk1ODI5IiwiYWRkcmVzc0xpbmUxIjoiODkxNSBHZXJiZXIgUm9hZCIsImNpdHkiOiJTYWNyYW1lbnRvIiwic3RhdGUiOiJDQSIsImNvdW50cnkiOiJVUyIsInBvc3RhbENvZGU5IjoiOTU4MjktMDAwMCJ9LCJnZW9Qb2ludCI6eyJsYXRpdHVkZSI6MzguNDgyNjc3LCJsb25naXR1ZGUiOi0xMjEuMzY5MDI2fSwiaXNHbGFzc0VuYWJsZWQiOnRydWUsInNjaGVkdWxlZEVuYWJsZWQiOnRydWUsInVuU2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZX1dLCJkZWxpdmVyeSI6eyJidUlkIjoiMCIsIm5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJub2RlVHlwZSI6IlNUT1JFIiwiYWRkcmVzcyI6eyJwb3N0YWxDb2RlIjoiOTU4MjkiLCJhZGRyZXNzTGluZTEiOiI4OTE1IEdlcmJlciBSb2FkIiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeSI6IlVTIiwicG9zdGFsQ29kZTkiOiI5NTgyOS0wMDAwIn0sImdlb1BvaW50Ijp7ImxhdGl0dWRlIjozOC40ODI2NzcsImxvbmdpdHVkZSI6LTEyMS4zNjkwMjZ9LCJpc0dsYXNzRW5hYmxlZCI6dHJ1ZSwic2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZSwidW5TY2hlZHVsZWRFbmFibGVkIjp0cnVlLCJhY2Nlc3NQb2ludHMiOlt7ImFjY2Vzc1R5cGUiOiJERUxJVkVSWV9BRERSRVNTIn1dfSwic2hpcHBpbmdBZGRyZXNzIjp7ImxhdGl0dWRlIjozOC40NzM4LCJsb25naXR1ZGUiOi0xMjEuMzQzOSwicG9zdGFsQ29kZSI6Ijk1ODI5IiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeUNvZGUiOiJVU0EiLCJnaWZ0QWRkcmVzcyI6ZmFsc2V9LCJhc3NvcnRtZW50Ijp7Im5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJhY2Nlc3NQb2ludHMiOm51bGwsImludGVudCI6IlBJQ0tVUCIsInNjaGVkdWxlRW5hYmxlZCI6ZmFsc2V9LCJpbnN0b3JlIjpmYWxzZSwicmVmcmVzaEF0IjoxNjM4OTQ5MzM0NzU4LCJ2YWxpZGF0ZUtleSI6InByb2Q6djI6ZTU3YThiNGMtZGY1My00NjgyLTg1YTktYTBmNmNmNDE3ZWQxIn0=; bstc=Yg5DtHgPVqfBKTBt5d5n6o; _pxhd=aJdlPT71A7ycnnOG7ZfG-JjMdZSyroBkwuloPmd-mm3HPXlZ4gPymvTh6OaC3MlpeoniiPgPi93XO1NV6IQGrQ==:m6C8FJ77HPMqpVdhMFC5pEOznwmvK6ui2i3PwM0kEjFIgeRKLjRQOZxEfDzy9O-gKCsN5M5jS6dAMm4mDDZy6yT2ttQFhJigJ2ReJo6zih0=; mobileweb=0; xpa=CfaEu; exp-ck=CfaEu1; TB_DC_Flap_Test=0; auth=MTAyOTYyMDE4znGm2ANEIB+T4KvLdbeyV1Sfq0qx+cXWUfeIShm7S2W2P7GkCYi7ekc0IJ9jXWO5IHuvxUZkOnxePCTtK7wgyPmya3spLau9B5jiDkR7sHIqJEEhjLzfZ9dTkdF/5mvs767wuZloTfhm7Wk2KcjygsxZsyqqU4iLObNl+MfESvnQc6+rly/Wmpxkhp4dnNsfbkuRL4kVTD80vzBWjR4vs3UF6U3pxtFVj4U3Tq87ITIUMk70P8glgOEpLOprhDfMywI05adPtwc9/m5r1ONHR/jLHE7P3UfRYIXmEC4TikehSEYvBHadkt0+i2E6vC92xNTLWPivL0GcKN6yiYH/9V0jQSjXf8n29AFvDsw0RrnPiFxLW1B9D2pAiD15iVj4ggPATkFkcLQdoPTWSz22YJE5WBBdZBCyKnCQAR7o6eg=; xpm=1+1638944037+YeDTYcBVVJrmBJn6nfVFmc~+0; bm_mi=78887EA9AF8BB37C05C76376607B6478~L407Wwxwj5/lU5A0UyL/Rnz49wUHMTHvvTnMVVO9LdsxHpbL/tiFS3lPrveT6Igddge3dhXhkZVU1X66muzkKbvq/avho1/Th3yhub1D5suR1e0byePHbXclDyBxqbi2kDbRyjbhatMLySfqEOkZttw15OU3g7nC9KCYiFZqliYyGd57419AjyIMj30U6KAJY82hFoPic8ww3bz0+fGshU/S1qvjsUZilhvnhy9/CqMiUG2F1GO3RWlt5E77vweD6NO/4fFgFD6dtI8eufPcdiWaLqhIwg7OIltYX/1Ybp2oYfE4p51OLUvrEQd+JTlLcmF54suid+LrOjSGe5JXNgT+JQ8aCzU6CYTQRANmCkqd7abN/bV5tq5JOOEyPyZjF+sVaIeLzj3JvpeG0YngvxXvUpoyGO6lAt/JPLwyPX5geqoDFfo8gpzLVF4rr0ZY; tb-c30=eus2-t1; TS013ed49a=01538efd7c27addd2cb2f8df23ee80015b0c013a429b4a79b4ac21a881ba67cc53b4a2e1010a73029c73da791b3ab3783c6f490219; ak_bmsc=2E7620D798BAA908B2B89CB2E06FBCF4~000000000000000000000000000000~YAAQHy7AF/7WUTZ9AQAAdvevmA732n4sMhIhwvEFKlbawLHVq5YZXPdEZATtew+jiKMypxhejZWGSKfQXCAGPMD44W0gfYbhay1bClsHHfAhBv+aflOUhEphKkUOEbwCLtA5SzBTHy7D5kIOiECklIhc+kmMln9DU+uWHBNwvJt3FL2ivsJvE8u++2UIO2koOUFaDolwQqm4+ZLjEip0T/zqiRiA6CiP56ECo5cd8WLMLBoV2+74zEm246hDcF1fkscu+R2qiCLGRiZMyk2/3bLsyY6gVdzbhN6lEV5r5VRtqMKEGEplzVt5wcWBYfq6JLhT/TAEy04WJNWe+PZLiaMMfAKt+hRO/RV0bK9xAe7aw7IMnNvl+55rfdDWBAbMcA0osgdSLoF9R4uNXKiX+L8zWFotqp6diR7eiq/JUmcmz+dP6NDLKCz90fBt8KOSbcTmD3+JqCLGbJ1XH8RIyG0AsE9n2wEUu/kifj0V9AOFDZXxjxkUy/55tPBwK/hgUUsTuA8qlhBuES78FzbuC60TJUw674ykQQaaEaq3ssn9oGDFEzm/0hvdKR+X9leRs3NefdB94LFrlIoP+w==; TS01b0be75=01538efd7c7074210c7381311242b150b782df6139050dc7e7d2b3537eb23177de4cf9e5a667f105a66a977b0af31da995c06936d0; TBWL-94-glassItemSRL=cg9pbc9j9ue8:0:2ryt4rdhzg3fy:19w8q1upz21gm; akavpau_p2=1638945037~id=5cb916bfc2c1af5d12074ea18facf95c; TBWL-94-cartGQLSRL=cg9pbdfg7ujt:0:1cj7fc3f3raiy:32uu59621dfh9; com.wm.reflector="reflectorid:0000000000000000000000@lastupd:1638944440000@firstcreate:1637756663354"; xptwg=3424725372:7B760B30995A48:14389D3:C3F92C65:826B5ED9:72FD20A:; bm_sv=F243E6A5F45DE500B2A21F640A97B6D4~XgCJJgs1HygXv+PM15+BXOYVZG6ifi6V0O7SFSlKTWpTONxyornEZq/R2TV9hx5NHUMrQC9zN70hp4ifOe0KfqwaRoQokok84UMH0az4Fmi7rEDD8hLeAZ4bCYC8QXhZdDK3iVq/xino7DlwRdxXDQ/de0iottfB8P/rxfp/teo=; _px3=6ed74163e64089bf07751635bcda79afdbfeb199c4284afae863212565944331:H81dL9ScroFN6OoYlIfyNYyBtMOHdXmgj44RsMUBxKon0CrJ0Y7l0QwSOyZ+kITijORiW9XmcnazAx8CdCLqbQ==:1000:CHfSZfgroENCbBBLEiCgQdh5dk/+SIu75MaaQPFs/tv6SSaMLDm1k24lzxNWDE/oVmGbnBoAU/F23muBSZ65A66R278nrvhqvZs0rfGyfQ3Dh5Z2KaFnFZy6nYxVhnWf8oOqG8ByxdpbNTF1Kb7t0KVzfA2S4OOFoTGFWP/B81RsV/UYIkFJM10IJJgAGyc6JYyEOD/O7GNbXWEK47NZrg==; _pxde=7f51fde4e178d74710636c8bd19c8cb8f83ef6d9703aee7e34d672ccc149d922:eyJ0aW1lc3RhbXAiOjE2Mzg5NDQ1NDMyOTQsImZfa2IiOjAsImlwY19pZCI6W119',

    }
    list1 = 0
    for uurl in words():
        list1 += 1
        uurl = uurl.strip()
        print(f'第{list1}开始获取中！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！')
        print(uurl)
        # run(headers, uurl)
        try:
            run(headers, uurl)
        except Exception as g:
            try:
                yidong(uurl)
                run(headers, uurl)
            except Exception as g:
                continue


# headers = {
#         'Host': 'www.walmart.com',
#         'Connection': 'keep-alive',
#         'Cache-Control': 'max-age=0',
#         'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
#         'sec-ch-ua-mobile': '?0',
#         'Upgrade-Insecure-Requests': '1',
#         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36',
#         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
#         'Service-Worker-Navigation-Preload': 'true',
#         'Sec-Fetch-Site': 'same-origin',
#         'Sec-Fetch-Mode': 'navigate',
#         'Sec-Fetch-User': '?1',
#         'Sec-Fetch-Dest': 'document',
#         'Accept-Encoding': 'gzip, deflate, br',
#         'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
#         'Cookie': 'AID=wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1637050244515; ACID=e57a8b4c-df53-4682-85a9-a0f6cf417ed1; hasACID=true; locGuestData=eyJpbnRlbnQiOiJTSElQUElORyIsInN0b3JlSW50ZW50IjoiUElDS1VQIiwibWVyZ2VGbGFnIjpmYWxzZSwicGlja3VwIjp7Im5vZGVJZCI6IjMwODEiLCJ0aW1lc3RhbXAiOjE2MzcwNTAyNDQ2MzJ9LCJwb3N0YWxDb2RlIjp7InRpbWVzdGFtcCI6MTYzNzA1MDI0NDYzMiwiYmFzZSI6Ijk1ODI5In0sInZhbGlkYXRlS2V5IjoicHJvZDp2MjplNTdhOGI0Yy1kZjUzLTQ2ODItODVhOS1hMGY2Y2Y0MTdlZDEifQ==; vtc=YeDTYcBVVJrmBJn6nfVFmc; TBV=7; _pxvid=b2839e3b-46b4-11ec-a6e3-585a4b527450; assortmentStoreId=3081; pxcts=3ddb24a0-4e68-11ec-ba30-89fbf66c1aed; hasLocData=1; tb_sw_supported=true; adblocked=false; dimensionData=969; TB_Latency_Tracker_100=1; TB_Navigation_Preload_01=1; TB_SFOU-100=1; locDataV3=eyJpbnRlbnQiOiJTSElQUElORyIsInBpY2t1cCI6W3siYnVJZCI6IjAiLCJub2RlSWQiOiIzMDgxIiwiZGlzcGxheU5hbWUiOiJTYWNyYW1lbnRvIFN1cGVyY2VudGVyIiwibm9kZVR5cGUiOiJTVE9SRSIsImFkZHJlc3MiOnsicG9zdGFsQ29kZSI6Ijk1ODI5IiwiYWRkcmVzc0xpbmUxIjoiODkxNSBHZXJiZXIgUm9hZCIsImNpdHkiOiJTYWNyYW1lbnRvIiwic3RhdGUiOiJDQSIsImNvdW50cnkiOiJVUyIsInBvc3RhbENvZGU5IjoiOTU4MjktMDAwMCJ9LCJnZW9Qb2ludCI6eyJsYXRpdHVkZSI6MzguNDgyNjc3LCJsb25naXR1ZGUiOi0xMjEuMzY5MDI2fSwiaXNHbGFzc0VuYWJsZWQiOnRydWUsInNjaGVkdWxlZEVuYWJsZWQiOnRydWUsInVuU2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZX1dLCJkZWxpdmVyeSI6eyJidUlkIjoiMCIsIm5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJub2RlVHlwZSI6IlNUT1JFIiwiYWRkcmVzcyI6eyJwb3N0YWxDb2RlIjoiOTU4MjkiLCJhZGRyZXNzTGluZTEiOiI4OTE1IEdlcmJlciBSb2FkIiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeSI6IlVTIiwicG9zdGFsQ29kZTkiOiI5NTgyOS0wMDAwIn0sImdlb1BvaW50Ijp7ImxhdGl0dWRlIjozOC40ODI2NzcsImxvbmdpdHVkZSI6LTEyMS4zNjkwMjZ9LCJpc0dsYXNzRW5hYmxlZCI6dHJ1ZSwic2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZSwidW5TY2hlZHVsZWRFbmFibGVkIjp0cnVlLCJhY2Nlc3NQb2ludHMiOlt7ImFjY2Vzc1R5cGUiOiJERUxJVkVSWV9BRERSRVNTIn1dfSwic2hpcHBpbmdBZGRyZXNzIjp7ImxhdGl0dWRlIjozOC40NzM4LCJsb25naXR1ZGUiOi0xMjEuMzQzOSwicG9zdGFsQ29kZSI6Ijk1ODI5IiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeUNvZGUiOiJVU0EiLCJnaWZ0QWRkcmVzcyI6ZmFsc2V9LCJhc3NvcnRtZW50Ijp7Im5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJhY2Nlc3NQb2ludHMiOm51bGwsImludGVudCI6IlBJQ0tVUCIsInNjaGVkdWxlRW5hYmxlZCI6ZmFsc2V9LCJpbnN0b3JlIjpmYWxzZSwicmVmcmVzaEF0IjoxNjM4OTQ5MzM0NzU4LCJ2YWxpZGF0ZUtleSI6InByb2Q6djI6ZTU3YThiNGMtZGY1My00NjgyLTg1YTktYTBmNmNmNDE3ZWQxIn0=; bstc=Yg5DtHgPVqfBKTBt5d5n6o; _pxhd=aJdlPT71A7ycnnOG7ZfG-JjMdZSyroBkwuloPmd-mm3HPXlZ4gPymvTh6OaC3MlpeoniiPgPi93XO1NV6IQGrQ==:m6C8FJ77HPMqpVdhMFC5pEOznwmvK6ui2i3PwM0kEjFIgeRKLjRQOZxEfDzy9O-gKCsN5M5jS6dAMm4mDDZy6yT2ttQFhJigJ2ReJo6zih0=; mobileweb=0; xpa=CfaEu; exp-ck=CfaEu1; TB_DC_Flap_Test=0; auth=MTAyOTYyMDE4znGm2ANEIB+T4KvLdbeyV1Sfq0qx+cXWUfeIShm7S2W2P7GkCYi7ekc0IJ9jXWO5IHuvxUZkOnxePCTtK7wgyPmya3spLau9B5jiDkR7sHIqJEEhjLzfZ9dTkdF/5mvs767wuZloTfhm7Wk2KcjygsxZsyqqU4iLObNl+MfESvnQc6+rly/Wmpxkhp4dnNsfbkuRL4kVTD80vzBWjR4vs3UF6U3pxtFVj4U3Tq87ITIUMk70P8glgOEpLOprhDfMywI05adPtwc9/m5r1ONHR/jLHE7P3UfRYIXmEC4TikehSEYvBHadkt0+i2E6vC92xNTLWPivL0GcKN6yiYH/9V0jQSjXf8n29AFvDsw0RrnPiFxLW1B9D2pAiD15iVj4ggPATkFkcLQdoPTWSz22YJE5WBBdZBCyKnCQAR7o6eg=; xpm=1+1638944037+YeDTYcBVVJrmBJn6nfVFmc~+0; bm_mi=78887EA9AF8BB37C05C76376607B6478~L407Wwxwj5/lU5A0UyL/Rnz49wUHMTHvvTnMVVO9LdsxHpbL/tiFS3lPrveT6Igddge3dhXhkZVU1X66muzkKbvq/avho1/Th3yhub1D5suR1e0byePHbXclDyBxqbi2kDbRyjbhatMLySfqEOkZttw15OU3g7nC9KCYiFZqliYyGd57419AjyIMj30U6KAJY82hFoPic8ww3bz0+fGshU/S1qvjsUZilhvnhy9/CqMiUG2F1GO3RWlt5E77vweD6NO/4fFgFD6dtI8eufPcdiWaLqhIwg7OIltYX/1Ybp2oYfE4p51OLUvrEQd+JTlLcmF54suid+LrOjSGe5JXNgT+JQ8aCzU6CYTQRANmCkqd7abN/bV5tq5JOOEyPyZjF+sVaIeLzj3JvpeG0YngvxXvUpoyGO6lAt/JPLwyPX5geqoDFfo8gpzLVF4rr0ZY; tb-c30=eus2-t1; TS013ed49a=01538efd7c27addd2cb2f8df23ee80015b0c013a429b4a79b4ac21a881ba67cc53b4a2e1010a73029c73da791b3ab3783c6f490219; ak_bmsc=2E7620D798BAA908B2B89CB2E06FBCF4~000000000000000000000000000000~YAAQHy7AF/7WUTZ9AQAAdvevmA732n4sMhIhwvEFKlbawLHVq5YZXPdEZATtew+jiKMypxhejZWGSKfQXCAGPMD44W0gfYbhay1bClsHHfAhBv+aflOUhEphKkUOEbwCLtA5SzBTHy7D5kIOiECklIhc+kmMln9DU+uWHBNwvJt3FL2ivsJvE8u++2UIO2koOUFaDolwQqm4+ZLjEip0T/zqiRiA6CiP56ECo5cd8WLMLBoV2+74zEm246hDcF1fkscu+R2qiCLGRiZMyk2/3bLsyY6gVdzbhN6lEV5r5VRtqMKEGEplzVt5wcWBYfq6JLhT/TAEy04WJNWe+PZLiaMMfAKt+hRO/RV0bK9xAe7aw7IMnNvl+55rfdDWBAbMcA0osgdSLoF9R4uNXKiX+L8zWFotqp6diR7eiq/JUmcmz+dP6NDLKCz90fBt8KOSbcTmD3+JqCLGbJ1XH8RIyG0AsE9n2wEUu/kifj0V9AOFDZXxjxkUy/55tPBwK/hgUUsTuA8qlhBuES78FzbuC60TJUw674ykQQaaEaq3ssn9oGDFEzm/0hvdKR+X9leRs3NefdB94LFrlIoP+w==; TS01b0be75=01538efd7c7074210c7381311242b150b782df6139050dc7e7d2b3537eb23177de4cf9e5a667f105a66a977b0af31da995c06936d0; com.wm.reflector="reflectorid:0000000000000000000000@lastupd:1638944291000@firstcreate:1637756663354"; TBWL-94-cartGQLSRL=cg9p9henaaqr:0:1cj7fc3f3raiy:2mcqactg8j9yt; _px3=e0b68715ec0e656e94a31356acc2a97ce8b1569692172b1f30c5363c62a25c6e:rhRMoUXt8nNz5Ar44LunOdX8YDBXLrXmga1PMsJTAOxUpouM0x+lBhS6cUNCTjbOae5PDjOXxiUlXi76ZpGAxQ==:1000:lJYPPr88tpOcn3w43lPXbnVs/UI+TUNvV3T5Ss8P/0zaB0v2Aa7qxE1k1D/ou9wQmB59pTFe8rhY74j129KDJxtCGTJnAmsvZdFz/Uqp2gFhnJ+NNtw0IgSvEev7AzmafAuqDEFCqhogQr+04U2b4ztAPtIg30PqvR/ZPsmCNCxn3iwJWLghLh/nTipazKhI1v1dv4bLzStAZXyi4kOBSQ==; _pxde=b2de888fd4a873a5d5f2725cbb7ba6093bab04a96088f483413aad75fd74f0b5:eyJ0aW1lc3RhbXAiOjE2Mzg5NDQzNTcwNzIsImZfa2IiOjAsImlwY19pZCI6W119; xptwg=2897124631:E1AEBA4FF891D0:24F6AAC:C8C5D08B:1925981A:1A1C8311:; TBWL-94-glassItemSRL=cg9pbakbhuiy:0:2ryt4rdhzg3fy:3sntpz3oxrbic; akavpau_p2=1638945034~id=f5e007a7db744fb288c469fd579d87be; bm_sv=F243E6A5F45DE500B2A21F640A97B6D4~XgCJJgs1HygXv+PM15+BXOYVZG6ifi6V0O7SFSlKTWpTONxyornEZq/R2TV9hx5NHUMrQC9zN70hp4ifOe0KfqwaRoQokok84UMH0az4Fmg9s6wWiHb6Si1Aws//rlnaf1b+MzGAngriNL1jxtegR6vaCixUgR+zJ5Ly5k91m1I=https://www.walmart.com/ip/Bellella-Womens-Cute-Summer-Short-Sleeve-Solid-Print-V-Neck-Pajamas-Set-Pjs-Sets-Loungewear-Sleepwear-Shorts-Sets/279880435'
#
#     }

def run(headers, uurl):#headers, uurl
    # uurl = 'https://www.walmart.com/ip/30oz-Tumblers-Mug-Water-Bottle-Insulated-Stainless-Steel-Coffee-Mug-Drink-Eco-Friendly-Reusable-Bottle-Outdoor-Sport/360710932'
    response = requests.get(uurl, headers=headers, verify=False).content.decode()
    # print('response1', response)
    er=json.loads(re.findall('"idml":(.*?),"reviews"',response)[0])
    # print(er["longDescription"])
    selector1 = parsel.Selector(er["longDescription"])
    texts2 = str(selector1.css("ul >li::text").getall()).replace("'",'"')
    print(texts2)


    res = etree.HTML(response)
    res_panduan = res.xpath(
        "//div[@class='flex flex-column flex-auto relative z-0']/div[@class='flex-auto relative z-1']/div[@class='center']/div[@class='flex flex-column items-center justify-center pa3']/span[@class='w_Bc w_Bd mb6 tc']")
    # print(res_panduan)

    if res_panduan != []:
        print('This page could not be found.')
    else:
        src = res.xpath('//img/@src')  # 图片链接
        # print(src,type(src))
        str0 = ''
        for i in src:
            if i[:5] == 'https' and '?odnHeight=80&odnWidth=80&odnBg=FFFFFF' in i:
                str0 += '|' + i + '\n'
            # print(i)
        str2 = str0.replace('?odnHeight=80&odnWidth=80&odnBg=FFFFFF', '')
        # print(str2)
        color_size = res.xpath('//div[@class="w-100 f6 mt2"]/div[@class="mb2 mid-gray"]/span')
        try:
            color_lei = color_size[0].text
            color = color_size[1].text  # 颜色
        except Exception as g:
            color_lei = None
            color = None
        try:
            size1_lei = color_size[2].text
            size1 = color_size[3].text  # 尺寸
        except Exception as g:
            size1_lei = None
            size1 = None
        # print(color, size1)
        try:
            load = res.xpath('//script')
            # print(load[4].text,type(load[4]))
            str1 = ''
            for a in json.loads(load[4].text)["itemListElement"]:
                # print(a["item"]["name"])
                str1 += a["item"]["name"] + '/'
            # print(str1)  # 路径
        except Exception as g:
            str1 = 'None'
        title = res.xpath('//h1[@class="f3 b lh-copy dark-gray mt1 mb2"]')[0].text
        # print(title)  # 标题
        price = res.xpath('//span[@itemprop="price"]')[0].text
        # print(price)  # 价格
        # print(response)
        description = res.xpath('//script')

        description_all = ''
        # print(description)
        for e1 in description:
            # print(e1.text)
            if 'description' in str(e1.text):
                # print(e1.text)
                description1 = '<table>' + json.loads(e1.text)['description'] + '</table>'
                # print(description1)
                selector = parsel.Selector(description1)
                texts = selector.css("table ::text").getall()
                # print(texts)

                for text1 in texts:
                    description_all += text1.strip() + '\n'
                    # print(text1.strip().replace(' ',''))
                print(description_all)
                break
        # print(description_all)

        itmeid = uurl.split('/')[-1]
        # print(itmeid)

        # print('*'*30)
        load1 = res.xpath('//script')
        for e in load1:
            # print(e.text)

            if 'specifications' in str(e.text):
                shuxing2 = json.loads(re.findall('specifications":(.*?),"warnings', e.text)[0])
                # print(shuxing2)
                SalePrice = price
                Title = title
                Specifications = shuxing2
                if color == None and size1 == None:
                    VariationSpecific = ''
                elif color == None:

                    VariationSpecific = [
                        {
                            "name": f"{size1_lei}",
                            "value": f"{size1}"
                        }
                    ]
                elif size1 == None:
                    VariationSpecific = [
                        {
                            "name": f"{color_lei}",
                            "value": f"{color}"
                        }
                    ]
                else:
                    VariationSpecific = [
                        {
                            "name": f"{color_lei}",
                            "value": f"{color}"
                        },
                        {
                            "name": f"{size1_lei}",
                            "value": f"{size1}"
                        }
                    ]
                CategoryPath = str1[:-1]
                Images = str2[1:]
                SalePrice = str(SalePrice)
                Title = str(Title)
                Specifications = str(json.dumps(Specifications))
                VariationSpecific = str(json.dumps(VariationSpecific))
                CategoryPath = str(CategoryPath)
                Images = str(Images)
                Description = str(description_all)
                itmeid = f"{itmeid}".strip()
                mysql(SalePrice, Title, Specifications, VariationSpecific, CategoryPath,
                      Images, Description, itmeid,texts2)


def mysql(SalePrice, Title, Specifications, VariationSpecific, CategoryPath, Images, Description, itmeid,texts2):
    CrawlerTime = time.strftime('%Y/%m/%d %H:%M:%S')
    CrawlerStatus = '1'
    # sql = 'updata walmart_lunhui set SalePrice=%s,Title=%s,Specifications=%s,VariationSpecific=%s,CategoryPath=%s,Images=%s,CrawlerTime=%s,Description=%s values(SalePrice,Title,Specifications,VariationSpecific,CategoryPath,Images,CrawlerTime,Description)'
    cursor.execute(
        "update t_bi_walmart_listing_crawler set SalePrice=%s,Title=%s,Specifications=%s,VariationSpecific=%s,CategoryPath=%s,Images=%s,CrawlerTime=%s,Description=%s,CrawlerStatus=%s,KeyFeatures=%s where ItemId=%s",
        [SalePrice, Title, Specifications, VariationSpecific, CategoryPath, Images, CrawlerTime, Description,
         CrawlerStatus,texts2, itmeid])

    conn.commit()
    print(SalePrice, '\n', Title, '\n', Specifications, '\n', VariationSpecific, '\n', CategoryPath, '\n',
          Images, '\n', Description, '\n', itmeid)


def all_xpath(uurl, headers):
    print(uurl)


if __name__ == '__main__':
    maga()
    # run()
